import numpy as np
import os
import re
import requests
from sklearn.feature_extraction.text import TfidfVectorizer

# 下载数据集
url = "http://qwone.com/~jason/20Newsgroups/20news-18828.tar.gz"
response = requests.get(url)
open("20news-18828.tar.gz", "wb").write(response.content)

# 解压数据集
import tarfile
with tarfile.open("20news-18828.tar.gz", "r:gz") as tar:
    tar.extractall()

# 读取数据
data_dir = "20news-18828"

# 加载所有文本文件
def load_data(data_dir):
    texts = []
    labels = []
    for category in os.listdir(data_dir):
        category_path = os.path.join(data_dir, category)
        if os.path.isdir(category_path):
            for file_name in os.listdir(category_path):
                with open(os.path.join(category_path, file_name), 'r', encoding='latin1') as file:
                    texts.append(file.read())
                    labels.append(category)
    return texts, labels

texts, labels = load_data(data_dir)

# 使用TF-IDF向量化器将文本数据转换为数值特征
vectorizer = TfidfVectorizer(stop_words='english', max_features=500)  # 限制特征数量，避免维度过高
X = vectorizer.fit_transform(texts)

# 将标签转换为数值（可以选择不使用标签，直接聚类）
from sklearn.preprocessing import LabelEncoder
label_encoder = LabelEncoder()
y = label_encoder.fit_transform(labels)

# K-means算法实现
def k_means(X, k, max_iters=100):
    # 随机初始化K个簇中心
    centers = X[np.random.choice(X.shape[0], k, replace=False)]
    
    for i in range(max_iters):
        # Step 1: 分配每个点到最近的簇
        distances = np.linalg.norm(X.toarray()[:, np.newaxis] - centers, axis=2)
        labels = np.argmin(distances, axis=1)
        
        # Step 2: 更新簇中心
        new_centers = np.array([X[labels == j].mean(axis=0) if np.any(labels == j) else centers[j] for j in range(k)])
        
        # 如果簇中心没有变化，则结束
        if np.allclose(centers, new_centers):
            print(f"Converged after {i+1} iterations.")
            break
        centers = new_centers

    return centers, labels

# 运行K-means
k = 20  # 设置簇数
centers, labels = k_means(X, k)

# 输出部分结果
print(f"Cluster centers: \n{centers}")
print(f"Assigned labels: \n{labels[:10]}")  # 输出前10个数据点的簇标签
